
from bs4 import BeautifulSoup, SoupStrainer
import pandas as pd
import cchardet
import seaborn as sns
import matplotlib.pyplot as plt
import re
import numpy as np
import scipy as sp
from selenium import webdriver 
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from tqdm.notebook import tqdm
from wordcloud import WordCloud
from nltk import word_tokenize, pos_tag
from nltk.sentiment import SentimentIntensityAnalyzer
from nltk.corpus import stopwords
from nltk.stem import WordNetLemmatizer
from spellchecker import SpellChecker
from nltk.corpus import wordnet
from nltk import download
import contractions
from langdetect import detect
from transformers import AutoTokenizer
from transformers import AutoModelForSequenceClassification
from scipy.special import softmax
from nltk.tokenize import RegexpTokenizer
from sklearn.metrics import f1_score, r2_score
import statsmodels.api
from statsmodels.miscmodels.ordinal_model import OrderedModel
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score
from sklearn.model_selection import cross_val_score
from sklearn.utils import shuffle
from sklearn.naive_bayes import CategoricalNB
from sklearn.naive_bayes import MultinomialNB
from sklearn.linear_model import SGDClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.svm import SVC
import torch
import warnings

warnings.filterwarnings("ignore")
tqdm.pandas()
download('vader_lexicon')
download('averaged_perceptron_tagger')
download('punkt')

[nltk_data] Downloading package vader_lexicon to
[nltk_data]     C:\Users\y\AppData\Roaming\nltk_data...
[nltk_data]   Package vader_lexicon is already up-to-date!
[nltk_data] Downloading package averaged_perceptron_tagger to
[nltk_data]     C:\Users\y\AppData\Roaming\nltk_data...
[nltk_data]   Package averaged_perceptron_tagger is already up-to-
[nltk_data]       date!
[nltk_data] Downloading package punkt to
[nltk_data]     C:\Users\y\AppData\Roaming\nltk_data...
[nltk_data]   Package punkt is already up-to-date!

True


#if true then data that takes long to collect will be loaded in immediately from files
#else data will be collected slowly the way it was originally done
run_quick = True


if not run_quick:
    #set up a headless chrome browser
    #headless meaning no UI so just operates using commands
    #a browser will make it practically impossible for Amazon to detect us
    chrome_options = Options()
    chrome_options.add_argument("--headless=new")
    driver = webdriver.Chrome(options=chrome_options)
    
    #initialize some links to crawl through
    links = [
            'https://www.amazon.com/2022-Apple-MacBook-Laptop-chip/dp/B0B3C2R8MP',
            'https://www.amazon.com/Apple-MacBook-Laptop-10%E2%80%91core-16%E2%80%91core/dp/B0BSHDVBXZ',
            'https://www.amazon.com/Lenovo-i7-1165G7-Processor-Graphics-82A6000BUS/dp/B08Z9T5HNM', 
            'https://www.amazon.com/SGIN-Windows-Celeron-Expandable-Storage/dp/B0BZS78JSQ',
            'https://www.amazon.com/HP-Computer-Processor-Graphics-SnowBell/dp/B08RPBG7G4',
            'https://www.amazon.com/Acer-SFX14-42G-R607-Creator-LPDDR4X-Backlit/dp/B09R8ZHZSD',
            'https://www.amazon.com/ASUS-Processor-NumberPad-Microsoft-L210MA-DB01/dp/B081V6W99V',
            'https://www.amazon.com/Samsung-Chromebook-Graphics-Celeron-Processor/dp/B08M2LB12T',
            'https://www.amazon.com/Apple-MacBook-13-inch-256GB-Storage/dp/B08N5M7S6K',
            'https://www.amazon.com/Alienware-M15-Gaming-Laptop-15-6-inch/dp/B09VQ3JWZ5',
            'https://www.amazon.com/MSI-i9-12900H-14-Core-GeForce-Thunderbolt/dp/B0BPN42FTQ',
            'https://www.amazon.com/Microsoft-Surface-Laptop-Touch-Screen-Matte-Black/dp/B08YX1WLVN'
            ]

    #limit how many links max you want to crawl through
    limit = 2_000

    #store scraped data in data list; one sample per laptop
    data = []

    #keep track of pages processed
    processed = set()

    #have progress bar to know program not stalling
    with tqdm(total=limit) as progress:
        #crawl through links
        for link in links:
            #update progress bar
            progress.update(1)

            #use selenium driver to get html rather than requests since selenium driver can bypass bot detection better
            driver.get(link)

            #obtain link's html content for tables only
            soup_tables = BeautifulSoup(driver.page_source, 'lxml', parseOnlyThese=SoupStrainer('table'))

            #find links in comparison table which suggests similar products
            for link in soup_tables.find_all("a"):
                #only want details page
                if '/dp/' in link['href']:
                    new_link = 'http://www.amazon.com' + link['href']
                    #prevent infinite crawling and repeat visits
                    #new_link not in links may cause an issue if we run out of unique links
                    #aka when the scraper doesn't go deep enough but so far no issue
                    if (len(links) < limit) and (new_link not in links):
                        links.append(new_link)
            
            #now try to get laptop specs
            try:
                name = driver.find_element(By.ID, "productTitle").text

                #might get the same sample from a different link so check if name is processed
                if name in processed:
                    continue

                processed.add(name)

                size = soup_tables.find("tr", class_="a-spacing-small po-display.size").\
                                find('td', class_='a-span9').text

                brand = soup_tables.find("tr", class_="a-spacing-small po-brand").\
                                 find('td', class_='a-span9').text

                memory = soup_tables.find("tr", class_=["a-spacing-small po-hard_disk.size", 
                                                          "a-spacing-small po-memory_storage_capacity"]).\
                                  find('td', class_='a-span9').text

                ram = soup_tables.find("tr", class_="a-spacing-small po-ram_memory.installed_size").\
                               find('td', class_='a-span9').text

                os = soup_tables.find("tr", class_="a-spacing-small po-operating_system").\
                              find('td', class_='a-span9').text

                cpu = soup_tables.find("tr", class_="a-spacing-small po-cpu_model.family").\
                              find('td', class_='a-span9').text

                gc = soup_tables.find("tr", class_="a-spacing-small po-graphics_description").\
                              find('td', class_='a-span9').text

                price = float((driver.find_element(By.CLASS_NAME, "a-price-whole").text +\
                               driver.find_element(By.CLASS_NAME, "a-price-fraction").text).replace(',',''))
            except:
                #if one of the feature values are missing (typically categorical), just skip the laptop
                #note though that this could lead to biases in the observations 
                
                #but the data will be imbalanced anyways (since amazon has more types of certain laptops than others)
                #so imputation using a features mode would also skew the data or even create false relationships
                
                #so ultimately i'd rather just leave out some observations, than ruin the integrity of the data
                continue

            #ignore laptops with no reviews otherwise can't run NLP models
            #this could also bias the observations but I can't impute reviews
            try:
                soup_reviews = BeautifulSoup(driver.find_element(By.ID, "cm-cr-dp-review-list").get_attribute('innerHTML'), 'lxml')
            except:
                continue

            #stores the reviews for this particular laptop   
            reviews = []

            #find all reviews + stars on front of product page
            for review in soup_reviews.find_all("div", {'data-hook':'review'}):
                try:
                    star = review.find("i", {'data-hook':'review-star-rating'}).text
                    comment = review.find("div", {'data-hook':'review-collapsed'}).text
                except:
                    #if no star/comment is available, do not add the review to the dataset
                    #since we cannot "impute" a review
                    
                    #i could impute a star,however, but I'm using the stars to train the models 
                    #so unless I use sentiment scores to impute the stars (invalid circular approach), 
                    #I could make the models perform worse
                    continue
                reviews.append((star, comment))



            sample = [name, size, brand, memory, ram, os, cpu, gc, price, reviews]
            data.append(sample)

    #combine the data into a dataframe
    df = pd.DataFrame(data, columns = ['name', 'size', 'brand', 'hhd', 'ram', 'os', 'cpu', 'graphics_card', 'price', 'reviews'])
else:
    #load data from file
    df = pd.read_csv('laptop4.csv', index_col=0, converters={'reviews': eval})


#make a copy of original dataset
df_tidy = df.copy()
#look at the format of the dataset
df_tidy.head()


#Some of the names are too long since they include some of the technical specs of the laptop so lets split on ',' or ':'
df_tidy['name'] = df_tidy['name'].str.split(',|:').str[0]


#extract floating point number from size feature
#note size may have units at the end, they are stored in inches regardless
#(this may or may not be unique to US Amazon)
df_tidy['size'] = df_tidy['size'].str.extract(r'\s*(\d+\.?\d*)').astype('float')


#get the top10 laptop brands
#this will help prevent analyzing undersampled features
#also it will make analysis of results and models simpler
top10_brands = df_tidy.brand.str.strip().value_counts()[:10]


top10_brands

HP           352
ASUS         277
Lenovo       242
Acer         117
Dell          99
Microsoft     75
MSI           51
SAMSUNG       48
LG            23
Razer         16
Name: brand, dtype: int64


#label brands not in the top10 list as 'other'
df_tidy.brand = df_tidy.brand.apply(lambda brand: brand.strip() if brand.strip() in top10_brands.index else 'other')


#before we can work on cleaning the ram and hhd features we need to define the following functions
def translate_hhd_to_GB(s):
    x = int(re.search('\s*(\d+)', s).group(1))
    
    #units is gigabytes
    if 'gb' in s.lower():
        if x <= 8:
        #then gb was a mistag (no 8gb laptop)    
            return x * 1000
        else:
            return x
    #units is terabytes
    elif 'tb' in s.lower():
        if x > 8:
        #then tb was a mistage (no 8tb+ laptop)
            return x
        else:
            return x * 1000
            
    #no units must determine if tb or gb
    else:
        if x > 8:
            #is gb
            return x
        else:
            #is tb
            return x * 1000
        
def translate_ram_to_GB(s):
    x = int(re.search('\s*(\d+)', s).group(1))
    
    #units is gigabytes
    if 'gb' in s.lower():
        if x >= 1000:
        #then mistag (no 1tb ram laptop)
            return x / 1000
        else:
            return x
    #no units must determine if tb or gb
    elif 'mb' in s.lower():
        if x < 128:
        #then mistag (no 128 mb laptop)
            return x
        else:
            return x / 1000
    #no units must determine if gb or mb
    else:
        if x >= 1000:
            return x / 1000
        else:
            return x


#apply func to hhd
df_tidy.hhd = df_tidy.hhd.apply(translate_hhd_to_GB)


#Check for hhd outliers
df_tidy[(df_tidy.hhd > 6000) | (df_tidy.hhd < 16)]


#remove the outlier, note we could visit the amazon page and fix the data manually but this would ruin the "pipeline" flow
df_tidy = df_tidy[(df_tidy.hhd <= 6000) & (df_tidy.hhd >= 16)]


#check for ram outliers
df_tidy.ram = df_tidy.ram.apply(translate_ram_to_GB)


df_tidy[(df_tidy.ram > 128) | (df_tidy.ram < 2)]


#remoove the outliers
df_tidy = df_tidy[(df_tidy.ram <= 128) & (df_tidy.ram >= 2)]


#Fix column names
df_tidy.rename(columns={'size':'size_inches', 'hhd':'hhd_GB', 'ram':'ram_GB'}, inplace=True)


#Now extract the most popular os, cpu, and graphics card types and label the rest as "other"
#this process is similar to what we did for brand but involves much more regex
df_tidy.os.value_counts().head()

 Windows 11 Home     364
 Windows 10 Home     204
 Windows 11          202
 Chrome OS           183
 Windows 10          125
Name: os, dtype: int64


#extract the os names
df_tidy.os = df_tidy.os.str.lower().str.extract(r'.*(?:(windows)\s*(\d+)\s*(?:home|pro|s)?)|.*(chrome)|.*(mac)')\
                .fillna('').agg(' '.join, axis=1).str.strip().replace('', 'other')


df_tidy.graphics_card.value_counts().head()

 Integrated      1058
 Dedicated        209
 RTX 3060          19
 RTX 3070 Ti       11
 RTX 3050 Ti       10
Name: graphics_card, dtype: int64


#extract the gpu names
df_tidy.graphics_card = df_tidy.graphics_card.str.lower().str.extract\
                            (r'.*(integrated)|.*(dedicated)|.*(rtx)|.*(gtx)|.*(mx)|.*(intel)')\
                            .fillna('').agg(' '.join, axis=1).str.strip().replace('', 'other')


df_tidy.cpu.value_counts().head()

 Core i7           126
 Celeron            94
 Core i5            94
 Intel Core i5      70
 Intel Core i7      66
Name: cpu, dtype: int64


#extract the cpu names
df_tidy.cpu = df_tidy.cpu.str.lower().str.extract(r'.*(i\d)|.*(celeron)|.*(ryzen \d)|.*(amd)|.*(pentium)')\
                            .fillna('').agg(' '.join, axis=1).str.strip().replace('', 'other')


#fix the price so that the decimal point is 2 places left
#i couldve easily done this in the data scraping process but I forgot
df_tidy.price = df_tidy.price / 100


#check for outliers
df_tidy[(df_tidy.price > 50000) | (df_tidy.price < 20)]


#remove outliers
df_tidy = df_tidy[(df_tidy.price <= 50000) & (df_tidy.price >= 20)]


#split the reviews list into multiple rows with a unique (star, text) tuple (for each tuple in the list)
#explode makes this very convenient; its kind of like melting a dataset but for values instead of features\
#this will create a lot more rows/observations since now each observation is not just a laptop but a review for that laptop
df_tidy_exploded = df_tidy.explode('reviews').reset_index()


#Now create a new function that will extract the star rating and text from each 'reviews' tuple
#and add the rating and text as new features to replace the previous 'reviews' tuple
def split(r):
    rating, review = r
    rating = int(re.search(r'(\d)', rating).group(1))
    review = review.replace("\n", "")
    return pd.Series([rating, review], index=['rating', 'review'])


#apply the func
df_tidy_exploded = df_tidy_exploded.join(df_tidy_exploded.reviews.apply(split), how='left').drop( ['index', 'reviews'], axis=1)


#Now create a function that determines if a review is in English
#we want to remove non-english reviews so that our NLP models work better
def isEnglish(r):
    try:
        #check if first 50 words are english
        return detect(r[:50]) == 'en'
    except:
        return False


#apply the func
if not run_quick:
    df_tidy_exploded = df_tidy_exploded[df_tidy_exploded.review.progress_apply(isEnglish)].reset_index().drop(['index'], axis=1)
else:
    df_tidy_exploded = pd.read_csv('english_reviews_only', sep='\t', index_col=0)


#look at the sample of the final result
df_tidy_exploded.sample(5)


df_tidy_exploded.corr(numeric_only=True)


df_tidy.corr(numeric_only=True)


sns.pairplot(data=df_tidy)

<seaborn.axisgrid.PairGrid at 0x17a7be16a40>


plt.figure(figsize=(8,5))
ax = sns.histplot(data=df_tidy, x='price')
plt.title("Price Distribution")
plt.ylabel("Count")
plt.xlabel("Price")
plt.show()
print(f'Mean: {df_tidy.price.mean()}', f'Median: {df_tidy.price.median()}', f'STD: {df_tidy.price.std()}', sep='\n')

Mean: 732.835624123422
Median: 544.5
STD: 646.4337442875417


fig, axs = plt.subplots(1,1, figsize=(8,5))
np.random.seed(0)
log_price = np.log(df_tidy.price) 
#sample as many elements as are in log price from the normal distribution
standard_normal = np.random.standard_normal(size=log_price.size)

#plot the density of the standard normal with sample size = price's size
sns.histplot(x=standard_normal, kde=True, color='orange', label='Standard Normal')
#standardize the normal
sns.histplot(x=(log_price - log_price.mean()) / log_price.std(), ax=axs, label='Log Price Normalized')
axs.legend()
plt.show()


plt.figure(figsize=(8,5))
ax = sns.histplot(data=df_tidy_exploded, x='rating', discrete=True)
plt.title("Rating Distribution")
plt.ylabel("Count")
plt.xlabel("Star Rating")
plt.show()


fig, axs = plt.subplots(2,1, figsize=(10,10))
#use median as measure of y value
sns.barplot(data=df_tidy, x='brand', y='price', estimator=np.median, \
                 order=df_tidy.groupby('brand').price.median().sort_values().index.values, ax=axs[0])
#use mean as measure of y value
sns.barplot(data=df_tidy_exploded, x='brand', y='rating', estimator=np.mean, \
                 order=df_tidy.groupby('brand').price.median().sort_values().index.values, ax=axs[1])

axs[0].set_title('Brand vs Price')
axs[0].set(xlabel=None, ylabel='Median Price (USD)')
axs[1].set_title('Brand vs Star Rating')
axs[1].set(xlabel=None, ylabel='Mean Stars')
fig.supxlabel("Brand")

plt.show()


fig, axs = plt.subplots(2,1, figsize=(10,10))
sns.barplot(data=df_tidy, x='graphics_card', y='price', estimator=np.median, \
                 order=df_tidy.groupby('graphics_card').price.median().sort_values().index.values, ax=axs[0])

sns.barplot(data=df_tidy_exploded, x='graphics_card', y='rating', estimator=np.mean, \
                 order=df_tidy.groupby('graphics_card').price.median().sort_values().index.values, ax=axs[1])

axs[0].set_title('GPU vs Price')
axs[0].set(xlabel=None, ylabel='Median Price (USD)')
axs[1].set_title('GPU vs Star Rating')
axs[1].set(xlabel=None, ylabel='Mean Stars')
fig.supxlabel("Graphics Card")

plt.show()


fig, axs = plt.subplots(2,1, figsize=(10,10))
sns.barplot(data=df_tidy, x='hhd_GB', y='price', estimator=np.median, ax=axs[0])

sns.barplot(data=df_tidy_exploded, x='hhd_GB', y='rating', estimator=np.mean, ax=axs[1])

axs[0].set_title('HHD vs Price')
axs[0].set(xlabel=None, ylabel='Median Price (USD)')
axs[1].set_title('HHD vs Star Rating')
axs[1].set(xlabel=None, ylabel='Mean Stars')
fig.supxlabel("Gigabytes Storage")

plt.show()


fig, axs = plt.subplots(2,1, figsize=(10,10))
sns.barplot(data=df_tidy, x='ram_GB', y='price', estimator=np.median, ax=axs[0])

sns.barplot(data=df_tidy_exploded, x='ram_GB', y='rating', estimator=np.mean, ax=axs[1])

axs[0].set_title('RAM vs Price')
axs[0].set(xlabel=None, ylabel='Median Price (USD)')
axs[1].set_title('RAM vs Star Rating')
axs[1].set(xlabel=None, ylabel='Mean Stars')
fig.supxlabel("Gigabytes Storage")

plt.show()


fig, axs = plt.subplots(2,1, figsize=(14,10))
sns.barplot(data=df_tidy, x='size_inches', y='price', estimator=np.median, ax=axs[0])

sns.barplot(data=df_tidy_exploded, x='size_inches', y='rating', estimator=np.mean, ax=axs[1])

axs[0].set_title('Screen Size. vs Price')
axs[0].set(xlabel=None, ylabel='Median Price (USD)')
axs[1].set_title('Screen Size. vs Star Rating')
axs[1].set(xlabel=None, ylabel='Mean Stars')
fig.supxlabel("Inches")

plt.show()


plt.figure(figsize=(10,5))
ax = sns.barplot(data=df_tidy_exploded, x='rating', y='price', estimator=np.median)
plt.title("Rating Laptop Prices")
plt.ylabel("Median Price (USD)")
plt.xlabel("Rating")
plt.show()


#set the level of significance to the standard a = 0.05
level_of_sig = 0.05


#create a linear model y = x0 + c1x1
linear_price_rating = statsmodels.formula.api.ols(formula = 'rating ~ price', data = df_tidy_exploded).fit()
print(f'Price\'s p-value: {linear_price_rating.pvalues.price}')
print(f'Statistically significant: {linear_price_rating.pvalues.price < level_of_sig}')

Price's p-value: 0.013167722395347222
Statistically significant: True


#create a quadratic model y = x0 + c1(x1**2)
quadratic_price_rating = statsmodels.formula.api.ols(formula = 'rating ~ I(price**2)', data = df_tidy_exploded).fit()
print(f'Price**2\'s p-value: {quadratic_price_rating.pvalues[1]}')
print(f'Statistically significant: {quadratic_price_rating.pvalues[1] < level_of_sig}')

Price**2's p-value: 0.043883883413297284
Statistically significant: True


linear_price_rating.params.price * 1_000

-0.05315055698661708


z = np.log(df_tidy_exploded.price)
z = (z -z.mean())/z.std()
df_tidy_exploded['zprice'] = z


#create a linear model y = x0 + c1x1
linear_zprice_rating = statsmodels.formula.api.ols(formula = 'rating ~ zprice', data = df_tidy_exploded).fit()
print(f'Normalized Price\'s p-value: {linear_zprice_rating.pvalues.zprice}')
print(f'Statistically significant: {linear_zprice_rating.pvalues.zprice < level_of_sig}')

Normalized Price's p-value: 0.057311370714363284
Statistically significant: False


linear_size_rating = statsmodels.formula.api.ols(formula = 'rating ~ size_inches', data = df_tidy_exploded).fit()
print(f'Size\'s p-value: {linear_size_rating.pvalues.size_inches}')
print(f'Statistically significant: {linear_size_rating.pvalues.size_inches < level_of_sig}')

Size's p-value: 0.40560842955894705
Statistically significant: False


quadratic_size_rating = statsmodels.formula.api.ols(formula = 'rating ~ I(size_inches**2)', data = df_tidy_exploded).fit()
print(f'Size**2\'s p-value: {quadratic_size_rating.pvalues[1]}')
print(f'Statistically significant: {quadratic_size_rating.pvalues[1] < level_of_sig}')

Size**2's p-value: 0.3445732192303693
Statistically significant: False


df_tidy_exploded['centered_size'] = df_tidy_exploded.size_inches - df_tidy_exploded.size_inches.mean()


quadratic_csize_rating = statsmodels.formula.api.ols(formula = 'rating ~ I(centered_size**2)', data = df_tidy_exploded).fit()
print(f'Centered Size**2\'s p-value: {quadratic_csize_rating.pvalues[1]}')
print(f'Statistically significant: {quadratic_csize_rating.pvalues[1] < level_of_sig}')

Centered Size**2's p-value: 0.34946472241303794
Statistically significant: False


#create vader model
vader = SentimentIntensityAnalyzer()


#calculate sentiment scores
scores = df_tidy_exploded.review.apply(lambda s: vader.polarity_scores(s).values())


df_tidy_vader = df_tidy_exploded.join(pd.DataFrame(scores.to_list(), columns=['neg', 'neu', 'pos', 'compound']))


sns.histplot(data=df_tidy_vader, x='compound')
plt.title("Vader Compound Score Distribution")
plt.ylabel("Count")
plt.xlabel("Compound Score")
plt.show()


#set thresholds for classification step
neg_threshold = -0.5
pos_threshold = 0.5


#classify according to vader compound score
df_tidy_vader['vader_class'] = df_tidy_vader.compound.apply(lambda x: 'pos' if x >= pos_threshold else 'neg' if x <= neg_threshold else 'neu')


sns.histplot(data=df_tidy_vader, x='vader_class')
plt.title("Vader Class Distribution")
plt.ylabel("Count")
plt.xlabel("Sentiment Classification")
plt.show()


df_tidy_vader.rating.value_counts()

5    4125
4    2128
3     814
1     543
2     354
Name: rating, dtype: int64


#create a word cloud that allows bi-grams (words found in pairs) and removes stopwords automatically
wc = WordCloud(collocation_threshold = 1, collocations = True)


#take a sample so it doesn't take too long to generate the clouds
positive_reviews = df_tidy_vader[df_tidy_vader.vader_class == 'pos'].review.sample(10)
wc.generate(positive_reviews.str.cat(sep=' '))
plt.imshow(wc, interpolation='bilinear')
plt.axis("off")
plt.show()


#define functions to clean text and remove nouns from text
tokenizer = RegexpTokenizer(r'\w+')
lemmatizer = WordNetLemmatizer()
speller = SpellChecker()
def clean(s):
    #lowercase
    s = s.lower()
    
    #fix mispellings
    spelled = [speller.correction(w) for w in word_tokenize(s) if speller.correction(w) != None]
    
    #expand contractions
    expanded = contractions.fix(" ".join(spelled))
    
    #remove links
    s = re.sub(r'http\S+','', s)
    #remove numbers
    s = re.sub(r'\d+', '', s)

    #remove nouns, punctuation, and lemmatize
    tokens = []
    for tok, pos in pos_tag(tokenizer.tokenize(expanded)):
        if pos[0] != "N":
            tokens.append(lemmatizer.lemmatize(tok, get_wordnet_pos(pos)))
            
    
    #remake the sentence
    sentence = " ".join(tokens)
    return sentence

#return part of speech in correct format
def get_wordnet_pos(treebank_tag):

    if treebank_tag.startswith('J'):
        return wordnet.ADJ
    elif treebank_tag.startswith('V'):
        return wordnet.VERB
    elif treebank_tag.startswith('R'):
        return wordnet.ADV
    else:
        return wordnet.NOUN


#clean the positive reviews
cleaned = positive_reviews.progress_apply(clean)
wc.generate(cleaned.str.cat(sep=' '))
plt.imshow(wc, interpolation='bilinear')
plt.axis("off")
plt.show()

  0%|          | 0/10 [00:00<?, ?it/s]


negative_reviews = df_tidy_vader[df_tidy_vader.vader_class == 'neg'].review.sample(10)
cleaned_neg = negative_reviews.progress_apply(clean)
wc.generate(cleaned_neg.str.cat(sep=' '))
plt.imshow(wc, interpolation='bilinear')
plt.axis("off")
plt.show()

  0%|          | 0/10 [00:00<?, ?it/s]


plt.figure(figsize=(10,5))
ax = sns.barplot(data=df_tidy_vader, x='rating', y='compound')
plt.title("Rating vs Vader Compound Score")
plt.ylabel("Average Compound Score")
plt.xlabel("Rating")
plt.show()


fig, axs = plt.subplots(1,3, figsize=(12,5))
sns.barplot(data=df_tidy_vader, x='rating', y='pos', ax=axs[0])
sns.barplot(data=df_tidy_vader, x='rating', y='neu', ax=axs[1])
sns.barplot(data=df_tidy_vader, x='rating', y='neg', ax=axs[2])
axs[0].set_title('Positive Score')
axs[0].set(xlabel=None, ylabel=None)
axs[1].set_title('Neutral Score')
axs[1].set(xlabel=None, ylabel=None)
axs[2].set_title('Negative Score')
axs[2].set(xlabel=None, ylabel=None)
fig.supxlabel("Rating")
fig.supylabel("Average Score")
plt.show()


#initialize the model
MODEL = f"cardiffnlp/twitter-roberta-base-sentiment-latest"
tokenizer = AutoTokenizer.from_pretrained(MODEL)
tokenizer.model_max_length = 512
model = AutoModelForSequenceClassification.from_pretrained(MODEL)

Some weights of the model checkpoint at cardiffnlp/twitter-roberta-base-sentiment-latest were not used when initializing RobertaForSequenceClassification: ['roberta.pooler.dense.bias', 'roberta.pooler.dense.weight']
- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).
- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).


#define the function that will run the model
def ROBERTA(r):
    #some sequences may have more than 512 tokens; just truncate here rather than seperate into chunks
    #chunking requires aggregation of token representations
    #also no pretrained longformers exist for sentiment analysis
    tokens = tokenizer(r, return_tensors='pt', truncation=True, padding='max_length') 
    output = model(**tokens) #unpack the tokens dict
    scores = output[0][0].detach().numpy() #turn tensor into np
    scores = softmax(scores) #normalize between 0 and 1
    return scores


#apply the model
if not run_quick:
    roberta_scores = pd.DataFrame(df_tidy_exploded.review.progress_apply(ROBERTA).to_list())
else:
    roberta_scores = pd.read_csv('roberta_scores2', sep='\t', index_col=0)
roberta_scores.columns = ['neg', 'neu', 'pos']


#merge the scores with the other meta-data
df_tidy_roberta = df_tidy_exploded.join(roberta_scores)


#look at the Roberta scores
fig, axs = plt.subplots(1,3, figsize=(12,5))
sns.barplot(data=df_tidy_roberta, x='rating', y='pos', ax=axs[0])
sns.barplot(data=df_tidy_roberta, x='rating', y='neu', ax=axs[1])
sns.barplot(data=df_tidy_roberta, x='rating', y='neg', ax=axs[2])
axs[0].set_title('Positive Score')
axs[0].set(xlabel=None, ylabel=None)
axs[1].set_title('Neutral Score')
axs[1].set(xlabel=None, ylabel=None)
axs[2].set_title('Negative Score')
axs[2].set(xlabel=None, ylabel=None)
fig.supxlabel("Rating")
fig.supylabel("Average Score")
plt.show()


#classify according to roberta scores
df_tidy_roberta['roberta_class'] = df_tidy_roberta[['neg','neu','pos']] \
    .apply(lambda x: 'pos' if x.pos == max(x.neg, x.neu, x.pos) else 'neg' if x.neg == max(x.neg, x.neu, x.pos) else 'neu', axis=1)


sns.histplot(data=df_tidy_roberta, x='roberta_class')
plt.title("Roberta Class Distribution")
plt.ylabel("Count")
plt.xlabel("Sentiment Classification")
plt.show()


#merge both model's scores
df_scores = df_tidy_vader[['neg','neu','pos', 'vader_class']].join(df_tidy_roberta[['neg','neu','pos', 'roberta_class']], lsuffix='_vader', rsuffix='_roberta')


df_scores.corr(numeric_only=True)


#merge the scores with the meta-data
complete_df = df_tidy_exploded.join(df_scores)


sns.pairplot(data=complete_df, palette='tab10', hue='rating', vars=['neg_vader', 'neu_vader', 'pos_vader',\
                                                                   'neg_roberta', 'neu_roberta', 'pos_roberta'])
plt.show()


#do an unbiased mapping
complete_df['actual_class_unbiased'] = complete_df.rating \
    .apply(lambda x: 'pos' if x in [4,5] else 'neg' if x in [1,2] else 'neu')


#do a very positively biased mapping
complete_df['actual_class_very_biased_pos'] = complete_df.rating \
    .apply(lambda x: 'pos' if x in [5,4,3] else 'neg' if x in [1] else 'neu')
complete_df.actual_class_very_biased_pos = pd.Categorical(complete_df.actual_class_very_biased_pos, ['pos','neu','neg'])


#do a verry negatively biased mapping
complete_df['actual_class_very_biased_neg'] = complete_df.rating \
    .apply(lambda x: 'pos' if x in [5] else 'neg' if x in [3,2,1] else 'neu')
complete_df.actual_class_very_biased_neg = pd.Categorical(complete_df.actual_class_very_biased_neg, ['pos','neu','neg'])


#do a slightly positive mapping
complete_df['actual_class_slightly_biased_pos'] = complete_df.rating \
    .apply(lambda x: 'pos' if x in [5,4] else 'neg' if x in [1] else 'neu')
complete_df.actual_class_slightly_biased_pos = pd.Categorical(complete_df.actual_class_slightly_biased_pos, ['pos','neu','neg'])


#do a slightly negative mapping
complete_df['actual_class_slightly_biased_neg'] = complete_df.rating \
    .apply(lambda x: 'pos' if x in [5] else 'neg' if x in [2,1] else 'neu')
complete_df.actual_class_slightly_biased_neg = pd.Categorical(complete_df.actual_class_slightly_biased_neg, ['pos','neu','neg'])


#plot the result distributions of the mappings along with the vader and roberta classification distributions
fig, axs = plt.subplots(2,4, figsize=(12,9), sharey=True)
sns.histplot(data=complete_df, x='actual_class_slightly_biased_pos', ax=axs[0][0])
sns.histplot(data=complete_df, x='actual_class_very_biased_pos', ax=axs[0][1])
sns.histplot(data=complete_df, x='actual_class_slightly_biased_neg', ax=axs[0][2])
sns.histplot(data=complete_df, x='actual_class_very_biased_neg', ax=axs[0][3])


sns.histplot(data=complete_df, x='actual_class_unbiased', ax=axs[1][0])
sns.histplot(data=complete_df, x='roberta_class', ax=axs[1][1])
sns.histplot(data=complete_df, x='vader_class', ax=axs[1][2])

axs[0][0].set_title('Weak Pos bias Stars\n(pos:5,4, neu:3,2, neg:1)')
axs[0][0].set(xlabel=None, ylabel=None)

axs[0][1].set_title('Strong Pos bias Stars\n(pos:5,4,3, neu:2, neg:1)')
axs[0][1].set(xlabel=None, ylabel=None)

axs[0][2].set_title('Weak Neg bias Stars\n(pos:5, neu:4,3, neg:2,1)')
axs[0][2].set(xlabel=None, ylabel=None)

axs[0][3].set_title('Strong Neg bias Stars\n(pos:5, neu:4, neg:3,2,1)')
axs[0][3].set(xlabel=None, ylabel=None)


axs[1][0].set_title('Balanced Stars\n(pos:5,4, neu:3, neg:2,1)')
axs[1][0].set(xlabel=None, ylabel=None)

axs[1][1].set_title('Roberta Prediction')
axs[1][1].set(xlabel=None, ylabel=None)

axs[1][2].set_title('Vader Prediction')
axs[1][2].set(xlabel=None, ylabel=None)

fig.supxlabel("Sentiment")
fig.supylabel("Count")
plt.show()


#choose weighted average for f1_score since we care about the imbalances of the data
#getting 10 positives wrong shouldn't be as bad as getting 10 negatives wrong since there are generally more positives
print('Roberta f1_score:', f1_score(complete_df.actual_class_unbiased, complete_df.roberta_class, average='weighted'))
print('Vader f1_score:', f1_score(complete_df.actual_class_unbiased, complete_df.vader_class, average='weighted'))

Roberta f1_score: 0.7567638410206838
Vader f1_score: 0.6907663380248565


print('Roberta f1_score:', f1_score(complete_df.actual_class_slightly_biased_pos, complete_df.roberta_class, average='weighted'))
print('Vader f1_score:', f1_score(complete_df.actual_class_slightly_biased_pos, complete_df.vader_class, average='weighted'))

Roberta f1_score: 0.7323512020765259
Vader f1_score: 0.6887340608216451


#store feature values in X
X = complete_df[['neg_roberta','neu_roberta','pos_roberta']]
#store true ratings in y
y = complete_df['rating']
#split sets into training and testing sets with test set having 40% of the data, and training use 60%
X_train, X_test, y_train, y_test = train_test_split(X,y,test_size=0.4, random_state=0)


#create linear regression model with roberta scores as indep and rating as dep
regression_model1 = statsmodels.formula.api.ols(formula='rating ~ pos_roberta + neg_roberta + neu_roberta', \
                                                data=X_train.join(y_train)).fit()


regression_model1_pred = regression_model1.predict(X_test)
regression_model1_residuals = y_test - regression_model1_pred


fig, axs = plt.subplots(1,2, figsize=(10,3))
sns.histplot(x=regression_model1_residuals, ax=axs[0])
sns.scatterplot(x=regression_model1_pred, y=regression_model1_residuals)

axs[0].set_title('Residuals distribution')
axs[0].set(xlabel='Residuals')

axs[1].set_title('Predicted vs Residuals')
axs[1].set(xlabel='Predicted', ylabel='Residuals')
plt.show()


#create ordinal regression model with roberta scores as indep and rating as dep
regression_model2 = OrderedModel.from_formula(formula='rating ~ pos_roberta + neu_roberta + neg_roberta',\
                                             data=X_train.join(y_train)).fit()


regression_model2_pred = pd.DataFrame(regression_model2.predict(X_test)).apply(lambda x: x.argmax() + 1, axis=1)


regression_model2_residuals = y_test - regression_model2_pred


fig, axs = plt.subplots(1,2, figsize=(10,3))
sns.histplot(x=regression_model2_residuals, ax=axs[0], discrete=True)
sns.scatterplot(x=regression_model2_pred, y=regression_model2_residuals)

axs[0].set_title('Residuals distribution')
axs[0].set(xlabel='Residuals')

axs[1].set_title('Predicted vs Residuals')
axs[1].set(xlabel='Predicted', ylabel='Residuals')
plt.show()


fig, axs = plt.subplots(2,1, figsize=(6,4), sharey=True)
sns.histplot(x=regression_model2_pred, discrete=True, ax=axs[0], label='Prediction', color='orange')
sns.histplot(x=y_test, discrete=True, ax=axs[1], label='Actual')

axs[0].set(xlabel=None, ylabel=None)
axs[1].set(xlabel='Stars', ylabel=None)
fig.supylabel("Count")
fig.legend()
plt.show()


print(f'Accuracy: {accuracy_score(y_test, regression_model2_pred)}')
print(f"f1_score: {f1_score(y_test, regression_model2_pred, average='weighted')}")

Accuracy: 0.5699937225360954
f1_score: 0.5184765059924523


# shuffle data first since not using training data anymore:
X_s, y_s = shuffle(X, y)


forest_clf = RandomForestClassifier()
forest_clf_f1_score = cross_val_score(forest_clf,X_s,y_s,cv=5,scoring='f1_weighted')
forest_clf_f1_accuracy = cross_val_score(forest_clf,X_s,y_s,cv=5,scoring='accuracy')
print(f'Forest average accuracy + std: {forest_clf_f1_score.mean(), forest_clf_f1_score.std()}')
print(f"Forest average f1_score + std: {forest_clf_f1_accuracy.mean(), forest_clf_f1_accuracy.std()}")

Forest average accuracy + std: (0.6362489883007191, 0.012472587273966877)
Forest average f1_score + std: (0.642140078925702, 0.011416872979547816)


sgd_clf = SGDClassifier()
sgd_clf_f1_score = cross_val_score(sgd_clf,X_s,y_s,cv=5,scoring='f1_weighted')
sgd_clf_f1_accuracy = cross_val_score(sgd_clf,X_s,y_s,cv=5,scoring='accuracy')
print(f'SGD average accuracy + std: {sgd_clf_f1_score.mean(), sgd_clf_f1_score.std()}')
print(f"SGD average f1_score + std: {sgd_clf_f1_accuracy.mean(), sgd_clf_f1_accuracy.std()}")

SGD average accuracy + std: (0.4429484949858483, 0.019069104167375363)
SGD average f1_score + std: (0.5641630153277373, 0.009305598052540772)


knn_clf = KNeighborsClassifier()
knn_clf_f1_score = cross_val_score(knn_clf,X_s,y_s,cv=5,scoring='f1_weighted')
knn_clf_f1_accuracy = cross_val_score(knn_clf,X_s,y_s,cv=5,scoring='accuracy')
print(f'KNN average accuracy + std: {knn_clf_f1_score.mean(), knn_clf_f1_score.std()}')
print(f"KNN average f1_score + std: {knn_clf_f1_accuracy.mean(), knn_clf_f1_accuracy.std()}")

KNN average accuracy + std: (0.5739867092032158, 0.013785076937675243)
KNN average f1_score + std: (0.5796084944496493, 0.015397911217188425)


svc_clf = SVC()
svc_clf_f1_score = cross_val_score(svc_clf,X_s,y_s,cv=5,scoring='f1_weighted')
svc_clf_f1_accuracy = cross_val_score(svc_clf,X_s,y_s,cv=5,scoring='accuracy')
print(f'Svc average accuracy + std: {svc_clf_f1_score.mean(), svc_clf_f1_score.std()}')
print(f"Svc average f1_score + std: {svc_clf_f1_accuracy.mean(), svc_clf_f1_accuracy.std()}")

Svc average accuracy + std: (0.5401244086919659, 0.010706283256139336)
Svc average f1_score + std: (0.5865144933708087, 0.010009329568392787)


categorical_NB_clf = CategoricalNB()
categorical_NB_clf_f1_score = cross_val_score(categorical_NB_clf,X_s,y_s,cv=5,scoring='f1_weighted')
categorical_NB_clf_accuracy = cross_val_score(categorical_NB_clf,X_s,y_s,cv=5,scoring='accuracy')
print(f'Categorical Naive Bayes average accuracy + std: {categorical_NB_clf_accuracy.mean(), categorical_NB_clf_accuracy.std()}')
print(f"Categorical Naive Bayes average f1_score + std: {categorical_NB_clf_f1_score.mean(), categorical_NB_clf_f1_score.std()}")

Categorical Naive Bayes average accuracy + std: (0.5179558337828503, 0.00013012330957993257)
Categorical Naive Bayes average f1_score + std: (0.35347306835477615, 0.0001473161873857265)


mult_NB_clf = MultinomialNB()
mult_NB_clf_f1_score = cross_val_score(mult_NB_clf,X_s,y_s,cv=5,scoring='f1_weighted')
mult_NB_clf_accuracy = cross_val_score(mult_NB_clf,X_s,y_s,cv=5,scoring='accuracy')
print(f'Multionmial Naive Bayes average accuracy + std: {mult_NB_clf_accuracy.mean(), mult_NB_clf_accuracy.std()}')
print(f"Multionmial Naive Bayes average f1_score + std: {mult_NB_clf_f1_score.mean(), mult_NB_clf_f1_score.std()}")

Multionmial Naive Bayes average accuracy + std: (0.5533652253735721, 0.006086941667609357)
Multionmial Naive Bayes average f1_score + std: (0.46292073992629845, 0.006797862289506721)


X2 = X.join(complete_df[['price']])
X2_s, y2_s = shuffle(X2, y)


forest_clf2 = RandomForestClassifier()
forest_clf2_f1_score = cross_val_score(forest_clf,X2_s,y2_s,cv=5,scoring='f1_weighted')
forest_clf2_f1_accuracy = cross_val_score(forest_clf,X2_s,y2_s,cv=5,scoring='accuracy')
print(f'2nd Forest average accuracy + std: {forest_clf2_f1_score.mean(), forest_clf2_f1_score.std()}')
print(f"2nd Forest average f1_score + std: {forest_clf2_f1_accuracy.mean(), forest_clf2_f1_accuracy.std()}")

2nd Forest average accuracy + std: (0.6412649880851655, 0.004148774482721585)
2nd Forest average f1_score + std: (0.656329197777967, 0.0037970935725663993)


X3 = X.join(complete_df[['price', 'size_inches', 'hhd_GB', 'ram_GB']])
X3_s, y3_s = shuffle(X3, y)


forest_clf3 = RandomForestClassifier()
forest_clf3_f1_score = cross_val_score(forest_clf,X3_s,y3_s,cv=5,scoring='f1_weighted')
forest_clf3_f1_accuracy = cross_val_score(forest_clf,X3_s,y3_s,cv=5,scoring='accuracy')
print(f'3rd Forest average accuracy + std: {forest_clf3_f1_score.mean(), forest_clf3_f1_score.std()}')
print(f"3rd Forest average f1_score + std: {forest_clf3_f1_accuracy.mean(), forest_clf3_f1_accuracy.std()}")

3rd Forest average accuracy + std: (0.6247668460235188, 0.010371430372930758)
3rd Forest average f1_score + std: (0.6443990195800093, 0.012699269009888277)


#fit the original forest model on all the data
forest_clf = forest_clf.fit(X,y)


#add some random sentences to test the predictions
str1 = "I HATE SCHOOOL. I SHOULDVE JOINED THE ARMY IM WASTING MY TIME OVER HERE"
str2 = "today is an awesome day. the sun is out and i'm loving it"
str3 = "the eiffel tower is very tall and made of metal. this is something not a lot of people know"
str4 = "this is quite sad, the day is almost over but at least im almost done with work"


#pass the unlabeled data to the forest model
print(f'{str1}: {forest_clf.predict([ROBERTA(str1)])[0]} stars', 
      f'{str2}: {forest_clf.predict([ROBERTA(str2)])[0]} stars', 
      f'{str3}: {forest_clf.predict([ROBERTA(str3)])[0]} stars', 
      f'{str4}: {forest_clf.predict([ROBERTA(str4)])[0]} stars', sep='\n')

I HATE SCHOOOL. I SHOULDVE JOINED THE ARMY IM WASTING MY TIME OVER HERE: 1 stars
today is an awesome day. the sun is out and i'm loving it: 5 stars
the eiffel tower is very tall and made of metal. this is something not a lot of people know: 5 stars
this is quite sad, the day is almost over but at least im almost done with work: 3 stars


#find 1 star ratings with high positive roberta sentiment
pos_sentiment_1stars = complete_df[complete_df.rating == 1].\
sort_values(by="pos_roberta", ascending=False)[['rating','pos_roberta','review']]


for i,r in pos_sentiment_1stars.head(2).iterrows():
    rating, score, review = r.values
    print(f'Stars: {rating}', f'Positive sentiment: {score}', review, sep='\n')

Stars: 1
Positive sentiment: 0.9778086
The laptop looks so beautiful, my first impression was the color. I opened up the laptop and set it up. The setup process was so smooth. The laptop is very fast for online browsing and social media I was very pleased with it. The audio quality is good and the touch screen works perfect. The weight and size are just perfect for a student to put in the backpack.Update...The laptop slowed down and was terrible, not recommended
Stars: 1
Positive sentiment: 0.9744418
Good product given the price. Works great! . Will update if anything comes up


#find 5 star ratings with high negative Roberta sentiment
neg_sentiment_5stars = complete_df[complete_df.rating == 5].sort_values(by="neg_roberta", ascending=False)[['rating','neg_roberta','review']]


for i,r in neg_sentiment_5stars.head(2).iterrows():
    rating, score, review = r.values
    print(f'Stars: {rating}', f'Negative sentiment: {score}', review, sep='\n')

Stars: 5
Negative sentiment: 0.95504034
A waste of money because no boot code! I am not happy! I want my money back but it’s outside my warranty. Sad state of affairs. don’t buy this item Surf pro… garbage! it’s a waste of your hard earned money.
Stars: 5
Negative sentiment: 0.9493004
My HP laptop recently got a virus!  I REALLY MISS IT!!!! I purchased a Chromebook laptop and I HATE IT!!!!!  Everything is so different and I know the OLD WAYS so Chromebook WILL NOT DO!!!!!!!


#normalize star ratings between 0 and 1
complete_df['norm_rating'] = (complete_df.rating - complete_df.rating.min()) / (complete_df.rating.max()  - complete_df.rating.min())


sns.histplot(data=complete_df, x='norm_rating')
plt.title('Distribution of Normalized Ratings')
plt.show()


def problem(weights, scores, desired_outcome):
    #take the weighted average of the sentiment scores
    combined = (scores * weights).apply(sum, axis=1)
    #noremalize the weighted average between 0 and 1
    normalized = (combined - combined.min())/(combined.max()-combined.min())
    #minimize the sum of squared errors between the normalized sentiment score and the normalized star ratings
    SSE = ((desired_outcome - normalized) ** 2).sum()
    return SSE


#bound weights between -1 and 1
bounds = sp.optimize.Bounds([-1.,-1.,-1.],[1.,1.,1.])
#bound sum of weights between 0 and 1
lc = sp.optimize.LinearConstraint([[1,1,1]], lb=0, ub=1)


#have an initial guess of neg_w = 0.1, neu_w = 0.3, pos_w = 0.6
opt_weights = sp.optimize.differential_evolution(problem, x0=[0.1,0.3,0.6], bounds=bounds, constraints=lc,
                                   args=(complete_df[['neg_roberta', 'neu_roberta', 'pos_roberta']], \
                                                     complete_df.norm_rating)).x


#apply weights
combined_score = (complete_df[['neg_roberta', 'neu_roberta', 'pos_roberta']]  * opt_weights).apply(sum, axis=1)
normalized_sentiment_score = (combined_score - combined_score.min()) / (combined_score.max() - combined_score.min())


fig, axs = plt.subplots(1,1, figsize=(8,5))
sns.histplot(x=normalized_sentiment_score,ax=axs, label='Combined+Normalized Sentiment Score')
sns.histplot(data=complete_df, x='norm_rating', label='Normalized Star Ratings')
plt.title('Distributions of Normalized Sentiment Metrics')
axs.set(xlabel='Score', ylabel='Count')
plt.legend()
plt.show()


#take the unweighted average
complete_df['sentiment_measure'] = (complete_df.norm_rating + normalized_sentiment_score)/2


#redo our previous analysis but with sentiment measure now
new_1star_pos = complete_df[complete_df.rating == 1].\
                sort_values(by="pos_roberta", ascending=False)[['rating','pos_roberta','sentiment_measure','review']]


for i,r in new_1star_pos.head(2).iterrows():
    rating, score, sentiment_measure, review = r.values
    print(f'Stars: {rating}', f'Roberta Positive sentiment: {score}', f'New Sentiment Measure: {sentiment_measure}', review, sep='\n')

Stars: 1
Roberta Positive sentiment: 0.9778086
New Sentiment Measure: 0.46207437202371904
The laptop looks so beautiful, my first impression was the color. I opened up the laptop and set it up. The setup process was so smooth. The laptop is very fast for online browsing and social media I was very pleased with it. The audio quality is good and the touch screen works perfect. The weight and size are just perfect for a student to put in the backpack.Update...The laptop slowed down and was terrible, not recommended
Stars: 1
Roberta Positive sentiment: 0.9744418
New Sentiment Measure: 0.46273585333631484
Good product given the price. Works great! . Will update if anything comes up


#redo our previous analysis but with sentiment measure now
new_5star_neg = complete_df[complete_df.rating == 5].\
                sort_values(by="neg_roberta", ascending=False)[['rating','neg_roberta','sentiment_measure','review']]


for i,r in new_1star_pos.head(2).iterrows():
    rating, score, sentiment_measure, review = r.values
    print(f'Stars: {rating}', f'Roberta Negative sentiment: {score}', f'New Sentiment Measure: {sentiment_measure}', review, sep='\n')

Stars: 1
Roberta Negative sentiment: 0.9778086
New Sentiment Measure: 0.46207437202371904
The laptop looks so beautiful, my first impression was the color. I opened up the laptop and set it up. The setup process was so smooth. The laptop is very fast for online browsing and social media I was very pleased with it. The audio quality is good and the touch screen works perfect. The weight and size are just perfect for a student to put in the backpack.Update...The laptop slowed down and was terrible, not recommended
Stars: 1
Roberta Negative sentiment: 0.9744418
New Sentiment Measure: 0.46273585333631484
Good product given the price. Works great! . Will update if anything comes up


#aggregate back to original format so that each sample is a laptop, 
#and take the average sentiment_measure of the laptop's reviews
samples = []
for name, x in complete_df.groupby('name'):
    sample = x[['name', 'size_inches', 'brand', 'hhd_GB', 'ram_GB',\
            'os', 'cpu', 'graphics_card', 'price', 'sentiment_measure']].iloc[0]
    sample['sentiment_measure'] = x.sentiment_measure.mean()
    samples.append(sample)

final_df = pd.DataFrame(samples).reset_index(drop=True)


#groupby features and sort by highest sentiment
print('Top 5 most loved laptops according to the new Sentiment Measure')
for i, name in enumerate(final_df.sort_values(by='sentiment_measure', ascending=False).head().name, start=1):
    print(f'\t{i}. {name}')

print('Top 5 most loved brands according to the new Sentiment Measure')
for i, name in enumerate(final_df.groupby(by='brand').sentiment_measure.mean().\
                         sort_values(ascending=False).head().index, start=1):
    print(f'\t{i}. {name}')
    
print('Top 5 most loved os according to the new Sentiment Measure')
for i, name in enumerate(final_df.groupby(by='os').sentiment_measure.mean().\
                         sort_values(ascending=False).head().index, start=1):
    print(f'\t{i}. {name}')
    
print('Top 5 most loved cpu according to the new Sentiment Measure')
for i, name in enumerate(final_df.groupby(by='cpu').sentiment_measure.mean().\
                         sort_values(ascending=False).head().index, start=1):
    print(f'\t{i}. {name}')

print('Top 5 most loved gpu according to the new Sentiment Measure')
for i, name in enumerate(final_df.groupby(by='graphics_card').sentiment_measure.mean().\
                         sort_values(ascending=False).head().index, start=1):
    print(f'\t{i}. {name}')

Top 5 most loved laptops according to the new Sentiment Measure
	1. Lenovo IdeaPad 3 Laptop Computer
	2. Razer Blade 15 Base Gaming Laptop 2020
	3. MSI 15.6" FHD 144Hz Gaming Laptop
	4. NewLenovo Student Business PC Notebook 14" HD Laptop Computer
	5. MSI GS65 Stealth-432 15.6" Gaming Laptop
Top 5 most loved brands according to the new Sentiment Measure
	1. MSI
	2. LG
	3. Acer
	4. ASUS
	5. Microsoft
Top 5 most loved os according to the new Sentiment Measure
	1. mac
	2. chrome
	3. other
	4. windows 10
	5. windows 11
Top 5 most loved cpu according to the new Sentiment Measure
	1. ryzen 3
	2. celeron
	3. ryzen 7
	4. amd
	5. other
Top 5 most loved gpu according to the new Sentiment Measure
	1. gtx
	2. dedicated
	3. intel
	4. integrated
	5. rtx


final_df.corr(numeric_only=True)

	name	size	brand	hhd	ram	os	cpu	graphics_card	price	reviews
0	Apple 2022 MacBook Air Laptop with M2 chip: 13...	13.6 Inches	Apple	256 GB	8 GB	Mac OS	Unknown	Integrated	104900.0	[(5.0 out of 5 stars, \nI am really happy with...
1	Apple 2023 MacBook Pro Laptop M2 Pro chip with...	14.2 Inches	Apple	512 GB	16 GB	Mac OS	Unknown	Integrated	194864.0	[(5.0 out of 5 stars, \nefore going into detai...
2	Lenovo IdeaPad Slim 7i 14.0" Laptop, 14.0" FHD...	14 Inches	Lenovo	512 GB	8 GB	Windows 10 Home	Intel Core i7	Integrated	66224.0	[(3.0 out of 5 stars, \nI use it almost every ...
3	SGIN Laptop 15.6 Inch 4GB DDR4 128GB SSD, Wind...	15.6 Inches	SGIN	128 GB	4 GB	Windows 11 Home	Celeron N4020	Integrated	35999.0	[(5.0 out of 5 stars, \nI bought this laptop s...
4	HP 2021 Stream 14" HD SVA Laptop Computer, Int...	14 Inches	HP	64 GB	4 GB	Windows 10 S	Celeron N	Integrated	28900.0	[(4.0 out of 5 stars, \nI bought this because ...

	size_inches	hhd_GB	ram_GB	price	rating
size_inches	1.000000	0.491069	0.416445	0.421194	-0.009320
hhd_GB	0.491069	1.000000	0.727549	0.693177	-0.029787
ram_GB	0.416445	0.727549	1.000000	0.681493	-0.024704
price	0.421194	0.693177	0.681493	1.000000	-0.027780
rating	-0.009320	-0.029787	-0.024704	-0.027780	1.000000

	size_inches	hhd_GB	ram_GB	price
size_inches	1.000000	0.473186	0.413983	0.409433
hhd_GB	0.473186	1.000000	0.738469	0.696047
ram_GB	0.413983	0.738469	1.000000	0.691577
price	0.409433	0.696047	0.691577	1.000000

	neg_vader	neu_vader	pos_vader	neg_roberta	neu_roberta	pos_roberta
neg_vader	1.000000	-0.107338	-0.337092	0.429730	0.065580	-0.373763
neu_vader	-0.107338	1.000000	-0.899846	0.261103	0.406359	-0.402013
pos_vader	-0.337092	-0.899846	1.000000	-0.435785	-0.413565	0.544661
neg_roberta	0.429730	0.261103	-0.435785	1.000000	0.180291	-0.882975
neu_roberta	0.065580	0.406359	-0.413565	0.180291	1.000000	-0.620921
pos_roberta	-0.373763	-0.402013	0.544661	-0.882975	-0.620921	1.000000

	size_inches	hhd_GB	ram_GB	price	sentiment_measure
size_inches	1.000000	0.462015	0.415949	0.418540	-0.021118
hhd_GB	0.462015	1.000000	0.723587	0.689339	-0.076979
ram_GB	0.415949	0.723587	1.000000	0.686657	-0.091338
price	0.418540	0.689339	0.686657	1.000000	-0.076666
sentiment_measure	-0.021118	-0.076979	-0.091338	-0.076666	1.000000

	name	size	brand	hhd	ram	os	cpu	graphics_card	price	reviews
141	SAMSUNG 15.6” Galaxy Book2 Pro with Intel ARC ...	15.6	SAMSUNG	1000	1	Windows	Core i7	Dedicated	189499.0	[(5.0 out of 5 stars, \nGreat picture quality,...
856	ASUS ZenBook S Ultra Slim Laptop	13.9	ASUS	1000	1	Windows 10 Pro	Intel Core i7	Intel Iris Xᵉ	169999.0	[(4.0 out of 5 stars, \nI loved this laptop.Ov...
1117	Microsoft Surface Pro 3 (256 GB	12.0	Microsoft	256	256	Windows 10 Pro	Core i5	Integrated	19995.0	[(4.0 out of 5 stars, \nI'll start with this: ...

	name	size_inches	brand	hhd_GB	ram_GB	os	cpu	graphics_card	price	rating	review
7552	HP 2022 Newest Pavilion 15.6" FHD 1080P IPS La...	15.6	HP	1000	32	windows 11	ryzen 7	other	843.22	5	This is a great PC for the money. Extremely fa...
5622	Acer Chromebook Spin 311 CP311-2H-C3KA Convert...	11.6	Acer	64	4	chrome	celeron	integrated	153.93	3	Product arrived without any physical blemishes...
6749	ASUS VivoBook 15 Thin and Light Laptop	15.6	ASUS	1000	4	windows 10	celeron	integrated	275.00	5	Exactly what I expected.Very nice laptop. Had...
5056	Dell Inspiron 13 5000 5310 Premium Laptop I 13...	13.3	Dell	512	16	windows 11	i5	integrated	607.22	1	The computer was working just fine for about 4...
5576	Lenovo Legion 5 15.6"	15.6	Lenovo	512	8	windows 11	ryzen 5	integrated	749.00	5	thank you so much! Very fast delivery and good...

CMSC320 Final Tutorial: Amazon Laptop Reviews Sentiment Analysis¶

By Andrew Widjaja¶

Introduction¶

Step 1: Intialization¶

Import Libraries¶

Enable or Disable 'run_quick'¶

Step 2: Data Collection¶

❗❗Here are some product specifications you can find in the above url❗❗

❗❗Here are some related links you can find in the above url❗❗

❗❗Here is a review you can find in the above url❗❗

Scrape Amazon Laptop Reviews + Metadata¶

Step 3: Clean + Tidy The Data 😱¶

❗❗The first 5 rows of our dataset❗❗

❗❗The top 10 laptop brands of our dataset❗❗

❗❗This outlier is invalid so let's remove it by conditioning the feature❗❗

❗❗These outliers are also invalid so let's remove it by conditioning the feature❗❗

❗❗Some of the top OS of our dataset❗❗

❗❗Some of the top graphics cards of our dataset❗❗

❗❗Some of the top CPUs of our dataset❗❗

❗❗A sample of the cleaned and tidied dataset❗❗

Step 4: Exploratory Data Analysis + Visualization 🔥¶

❗❗A correlation matrix between the numeric features of the exploded data set❗❗

❗❗A correlation matrix between the numeric features of the original data set❗❗

❗❗Plots of the relationships between numeric features and their distributions❗❗

❗❗Looks like price is related to brand, but brand is not related to star rating, though it is interesting to see that the most expensive is the lowest rated❗❗

❗❗Looks like price is related to GPU, but GPU is not related to star rating❗❗

❗❗Looks like price is positively related to HHD, but HHD is not really related to star rating❗❗

❗❗Looks like price is positively related to RAM, but RAM is not really related to star rating❗❗

❗❗Looks like price is positively related with screen size,also you could say that stars is related to size at the extreme ends,i.e. too small => decrease in rating, too large => decrease in rating,perhaps this indicates a quadratic relationship❗❗

❗❗Looks like price is not really related to rating, at least linearly visually, maybe quadratically?❗❗

Step 5: ML + Hypothesis Testing 🤖💡¶

Hypothesis testing¶

❗❗Linear regression model of Price vs Rating p-values❗❗

❗❗Quadratic regression model of Price vs Rating p-values❗❗

❗❗For every \$1,000 increase in price, rating goes down by 0.05 on average❗❗

❗❗Linear regression model of Normalized Price vs Rating p-values❗❗

❗❗Linear regression model of Screen size vs Rating p-values❗❗

❗❗Quadratic regression model of Screen size vs Rating p-values❗❗

❗❗Quadratic regression model of Centered Screen size vs Rating p-values❗❗

Vader Pretrained Bag of Words Model¶

❗❗Distribution of Vader compound scores❗❗

❗❗Distribution of Vader Sentiment classifications❗❗

❗❗Reminder of the distribution of stars❗❗

❗❗Word cloud of positive reviews determined by Vader❗❗

❗❗Word cloud of cleaned positive reviews determined by Vader❗❗

❗❗Word cloud of cleaned negative reviews determined by Vader❗❗

❗❗Seems like there's a linear positive relationship between rating and compound score❗❗

❗❗The positive and negative scores also display a linear relationship; neutral not so much❗❗

Roberta Pretrained transformer model¶

❗❗The positive and negative scores also display a linear relationship; and the neutral a quadratic one❗❗

❗❗Distribution of Roberta Sentiment classifications❗❗

Comparing the models¶

❗❗Comparing models' score distributions and correlations❗❗

Comparing Predicted Values to Actual Values¶

❗❗Comparing the distributions in classifcations produced by Roberta, Vader, and the different mappings of Stars to sentiment❗❗

❗❗F1_scores Assuming the balanced mapping is true❗❗

❗❗F1_scores Assuming the weak-pos bias mapping is true❗❗

Accomplishing Goal 1: Be able to predict a star rating from text¶

❗❗Analyzing residuals of simple linear regression model❗❗

❗❗Analyzing residuals of ordinal regression model❗❗

❗❗Analyzing performance of ordinal regression model❗❗

❗❗Analyzing performance of Random Forest❗❗

❗❗Analyzing performance of Stochastic Gradient Descent❗❗

❗❗Analyzing performance of K-Nearest Neighbors❗❗

❗❗Analyzing performance of Suppport Vector Classifier❗❗

❗❗Analyzing performance of Categorical Naive Bayes❗❗

❗❗Analyzing performance of Multinomial Naive Bayes❗❗

❗❗Analyzing performance of 2nd attempt at Random Forest❗❗

❗❗Analyzing performance of 3rd attempt at Random Forest❗❗

❗❗Testing the random forest model on unlabeled data❗❗

Goal 2: Combine the sentiment and star metrics into one robust metric¶

❗❗Analyzing disparities between stars and pos Roberta scores❗❗

❗❗Analyzing disparities between stars and neg Roberta scores❗❗

❗❗Result of attempting to fit combined+normalized sentiment score to the normalized star ratings❗❗

❗❗Sentiment measure found a happy medium between 1 star and high positive Roberta score❗❗

❗❗Sentiment measure found a happy medium between 5 star and high negative Roberta score❗❗

Step 6: Policy and Insight 💵💵¶

❗❗Laptops and features that make people happiest according to our new sentiment measure❗❗

Improvements¶

Sources¶